package com.zhang.spark_2.com.zhang.core.wc1

import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}

import scala.collection.mutable

/**
 * @title:
 * @author: zhang
 * @date: 2022/2/18 08:32 
 */
object WC_reduce {

  def main(args: Array[String]): Unit = {
    //TODO 获取环境
    val conf = new SparkConf().setMaster("local[*]").setAppName("WordCount")
    val sc = new SparkContext(conf)

    val data: RDD[String] = sc.makeRDD(List(
      "hello scala", "hello spark"
    ))

    val flatRdd: RDD[String] = data.flatMap(_.split(" "))

    val map: mutable.Map[String, Int] = flatRdd.map(word => mutable.Map((word, 1))).reduce(
      (map1, map2) => {
        map2.foldLeft(map1)(
          (map1, kv) => {
            var (word, cnt) = kv
            val oldCnt: Int = map1.getOrElse(word, 0)
            map1.update(word, oldCnt + cnt)
            map1
          }
        )
      }
    )
    println(map)
    sc.stop()

  }
}
